from lxml import etree
import csv
import os
import dao

con = None 

def dealWith(item:list):
    if len(item) > 0:
        return item[0].strip()
    else:
        return ""

def extract(wr)->None:
    for i in range(1,101):
        f = open(r'file'+os.sep+'{index}.html'.format(index=i),'r',encoding='utf-8')
        lz = f.readlines()
        text = '';
        for line in lz:
            text = text + line
        main_page = etree.HTML(text)
        main_lz = main_page.xpath('//*[@id="row_"]')
        for item in main_lz:
            name = item.xpath('td[3]/div[2]/a/text()')[0]
            sub_url = item.xpath('td[3]/div[2]/a/@href')[0]
            bg_id = sub_url.split('/')[-2]
            introduction = item.xpath('td[3]/p/text()')
            introduction = dealWith(introduction)
            geekRating = item.xpath('td[4]/text()')
            geekRating = dealWith(geekRating)
            avgRating = item.xpath('td[5]/text()')
            avgRating = dealWith(avgRating)
            numVoters = item.xpath('td[6]/text()')
            numVoters = dealWith(numVoters)
            lz = (bg_id,name,introduction,geekRating,avgRating,numVoters)
            wr.writerow(lz)
            dao.insert(lz,con)
        print(i,"done")
        f.close()



if __name__ == '__main__':
    con = dao.init()
    with open("result.csv","w",encoding='utf-8',newline='') as csvf:
        writer = csv.writer(csvf)
        extract(writer)
    dao.dispose(con)